Load libraries
library(magrittr)
library(tidyverse)
library(rvest)
library(plotly)
library(ggvis)
library(glue)
library(ggridges)
Scraping
Inspired by: https://www.analyticsvidhya.com/blog/2017/03/beginners-guide-on-web-scraping-in-r-using-rvest-with-hands-on-knowledge/
Selector Gadget (Chrome extension): http://selectorgadget.com/
get_html_pages <- function(urls) map(urls,read_html)
Civic, Corolla, Sentra, RJ, km < 80000, price < R$80000, year >= 2016, particular
olxURL <- function(brand,model,page,maxPrice=80000,maxKm=80000) glue("http://rj.olx.com.br/veiculos-e-pecas/carros/{brand}/{model}?f=p&me={maxPrice}&o={page}&pe={maxKm}&rs=34")
Construct URLs
urls <- map_chr(list(
list("honda","civic",1),
list("honda","civic",2),
list("toyota","corolla",1),
list("toyota","corolla",2),
list("toyota","corolla",3),
list("nissan","sentra",1),
list("nissan","sentra",2)),
~do.call(olxURL,.x))
# urls <- c("http://rj.olx.com.br/veiculos-e-pecas/carros/honda/civic?f=p&me=80000&pe=80000&rs=34"
# ,"http://rj.olx.com.br/veiculos-e-pecas/carros/toyota/corolla?f=p&me=80000&pe=80000&rs=34"
# ,"http://rj.olx.com.br/veiculos-e-pecas/carros/nissan/sentra?f=p&me=80000&pe=80000&rs=34"
# # 2nd page
# ,"http://rj.olx.com.br/veiculos-e-pecas/carros/honda/civic?f=p&me=80000&o=2&pe=80000&rs=34"
# ,"http://rj.olx.com.br/veiculos-e-pecas/carros/toyota/corolla?f=p&me=80000&o=2&pe=80000&rs=34"
# ,"http://rj.olx.com.br/veiculos-e-pecas/carros/nissan/sentra?f=p&me=80000&o=2&pe=80000&rs=34"
# # 3d page
# ,"http://rj.olx.com.br/veiculos-e-pecas/carros/toyota/corolla?f=p&me=80000&o=3&pe=80000&rs=34")
Get pages
pages <- get_html_pages(urls)
Auxiliary functions
get_html_href <- function(pages,css) unlist(map(pages,~(html_nodes(.x,css) %>% html_attr("href"))))
get_html_text <- function(pages,css) {
Links
text_links <- get_html_href(pages,".OLXad-list-link")
text_links %>% head
[1] "http://rj.olx.com.br/rio-de-janeiro-e-regiao/veiculos-e-pecas/carros/honda-civic-lxr-2016-aut-ipva-2018-pago-440723042"
[2] "http://rj.olx.com.br/rio-de-janeiro-e-regiao/veiculos-e-pecas/carros/honda-civic-lxr-2-0-flexone-carro-de-seguradora-leiam-455506916"
[3] "http://rj.olx.com.br/norte-do-estado-do-rio/veiculos-e-pecas/carros/honda-civic-2-0-lxr-2016-raridade-14-000-kms-455370116"
[4] "http://rj.olx.com.br/rio-de-janeiro-e-regiao/veiculos-e-pecas/carros/honda-civic-2-0-lxr-16v-flex-4p-automatico-455327176"
[5] "http://rj.olx.com.br/rio-de-janeiro-e-regiao/veiculos-e-pecas/carros/honda-civic-2016-impecavel-na-garantia-455314537"
[6] "http://rj.olx.com.br/rio-de-janeiro-e-regiao/veiculos-e-pecas/carros/honda-civic-lxr-2-0-flex-2016-417055450"
Car description
text_titles <- get_html_text(pages,".OLXad-list-title")
text_titles %>% head
[1] "Honda Civic LXR 2016 Aut IPVA 2018 PAGO - 2016"
[2] "Honda Civic LXR 2.0 Flexone/ Carro de Seguradora/ Leiam - 2016"
[3] "Honda Civic 2.0 LXR 2016 - Raridade (14.000 kms) - 2016"
[4] "Honda Civic 2.0 lxr 16v flex 4p automático - 2016"
[5] "Honda Civic 2016 Impecável (Na Garantia) - 2016"
[6] "Honda Civic LXR 2.0 Flex 2016 - 2016"
Preços
convertReais <- function(s) s %>%
str_replace_all("[^\\d]+","") %>%
as.integer()
text_prices <- get_html_text(pages,".OLXad-list-price") %>%
convertReais(.)
text_prices %>% head
[1] 68900 55000 72000 69900 59900 73800
Km, câmbio, motor: vindo assim: “33.000 km| Câmbio: automático| Flex”
clearMkCambioMotor <- function(s)
c(s[1] %>% str_replace("\\.","") %>% str_replace(" [kK]m",""),
s[2] %>% str_replace("Câmbio: ",""),
s[3])
decodeKmCambioMotor <- function(s) s %>%
str_replace_all("[\\n\\t]","") %>%
str_split("\\|") %>%
map(~str_trim(.x,"both")) %>%
map(~clearMkCambioMotor(.x))
text_km <- get_html_text(pages,".detail-specific") %>%
decodeKmCambioMotor(.)
[[3]]
[1] "14000" "automático" "Flex"
[[4]]
[1] "67000" "automático" "Flex"
[[5]]
[1] "59000" "automático" "Flex"
[[6]]
[1] "9000" "automático" "Flex"
Region
text_region <- get_html_text(pages,".detail-region") %>%
str_replace_all("[\\s\\n\\t]+"," ")
text_region %>% head
[1] "Rio de Janeiro, Grajaú - DDD 21" "Rio de Janeiro, Inhoaíba - DDD 21"
[3] "Macaé, Centro - DDD 22" "Rio de Janeiro, Anil - DDD 21"
[5] "Rio de Janeiro, Barra da Tijuca - DDD 21" "Rio de Janeiro, Del Castilho - DDD 21"
Montagem do DF
detectBrand <- function(s) case_when(str_detect(s,"corolla")~"corolla",
str_detect(s,"sentra")~"sentra",
str_detect(s,"civic")~"civic",
T~NA_character_)
df <- data_frame(title=str_sub(text_titles,end=-7),
brand=as.factor(detectBrand(str_to_lower(title))),
region=str_replace(text_region," - DDD \\d\\d",""),
ddd=str_sub(text_region,start=-2),
year=text_titles %>% str_sub(start=-4) %>% as.integer() %>% as.factor(),
price1k=round(text_prices/1000,1),
km1k=round(as.integer(map_chr(text_km,1))/1000,1),
cambio=as.factor(map_chr(text_km,2)),
Sentiment Analysis
Recursively get car description from linked page
# this will hit the server many times, use sparingly
pages_descr <- get_html_pages(df$link)
https://github.com/hadley/rvest/issues/175
#function definition
html_text_collapse <- function(x, trim = FALSE, collapse = "\n"){
UseMethod("html_text_collapse")
}
html_text_collapse.xml_nodeset <- function(x, trim = FALSE, collapse = "\n"){
vapply(x, html_text_collapse.xml_node, character(1), trim = trim, collapse = collapse)
}
html_text_collapse.xml_node <- function(x, trim = FALSE, collapse = "\n"){
paste(xml2::xml_find_all(x, ".//text()"), collapse = collapse)
}
Usando workaround do “html_text_collapse” pois html_text estava comendo os
tags:
html_nodes(pages_descr[[244]],".OLXad-description") %>% html_text_collapse %>% str_sub(end=400)
Versão do get_html_text com o workaround para inserir “” nos
get_html_text_collapse <- function(pages,css) {
text <- unlist(map(pages,~html_text_collapse(html_nodes(.x,css))))
text %>% str_trim("both")
}
Montando novo DF com descrcições (seguindo link do anuncio), limpeza
df_descr <- df %>% mutate(descr=unlist(get_html_text_collapse(pages_descr,".OLXad-description")),
model=unlist(get_html_text(pages_descr,".model")))
df_descr_clean <- df_descr %>%
mutate(key=row_number(),
descr=descr %>% str_to_lower() %>% str_replace_all("[^[:alnum:]/]"," ") %>%
str_replace_all("\\s{2,}"," "),
model=str_replace(model,"Modelo:[\\n\\t]+",""))
Reporta preço mediano e N por modelo. Parece q modelos são padronizados tipo FIPE
df_descr_clean %>%
group_by(model) %>%
summarize(price1k_median=median(price1k),N=n()) %>%
arrange(desc(price1k_median))
Para cada carro (identificado por “key”), liste as palavras usadas na descrição.
df_descr_words <- df_descr_clean %>%
select(key,descr) %>%
mutate(word=str_split(descr,"\\s+")) %>%
select(-descr) %>%
unnest() %>%
filter(str_length(word)>2,
!str_detect(word,"\\d"))
Estudo frequencial das palavras
df_descr_words_counted <- df_descr_words %>%
count(word,sort=T) %T>% print
Salvamos em excel para manualmente marcarmos palavras com sentimento negativo
df_descr_words_counted %>%
as.data.frame() %>%
xlsx::write.xlsx("df_descr_words_counted.xls", row.names=F)
Le arquivo com palavras com sentimento negativo
df_tretas <- read_csv("suspicious words.csv") %T>% print
Acha anuncios q contem palavra negativa
df_descr_words_negative <- df_descr_words %>%
semi_join(df_tretas,by="word") %>%
group_by(key) %>%
summarize(negativity_count=n(),
bad_words=paste0(word,collapse=",")) %T>%
print
Expande df de anúncios com análise de sentimento
df_descr_clean_sentiment <- df_descr_clean %>%
left_join(df_descr_words_negative,by="key") %>%
mutate(negativity_count=if_else(is.na(negativity_count),0L,negativity_count)) %>%
arrange(desc(negativity_count)) %T>%
print
Plots
df_filt <- df_descr_clean_sentiment %>%
filter(price1k>40,!is.na(brand),year==2016) %>%
df_filt %>%
ggplot(aes(x=negativity_count)) +
geom_bar(stat="count")

fun_length_y <- function(x) data.frame(y=median(x),label= paste0("N=", length(x)))
df_filt %>%
mutate(brand=fct_reorder(brand,price1k,.desc=T),
suspicious=negativity_count>1) %>%
arrange(suspicious) %>% # so true is drawn last
#mutate(suspicious=as.factor(suspicious) %>% fct_inorder() %>% fct_rev()) %>%
ggplot(aes(x=brand,y=price1k,fill=brand)) +
#geom_violin(alpha=.25) +

#theme(legend.position = "none")
df_filt_medians <- df_filt %>%
group_by(brand) %>%
summarize(N=n(),
price1k_median=median(price1k),
price1k_mean=mean(price1k)) %>%
arrange(desc(price1k_median)) %>%
mutate(y=row_number())
df_filt %>%
mutate(brand=fct_reorder(brand,price1k,.desc=T)) %>%
ggplot(aes(price1k, brand,fill=brand)) +
geom_density_ridges() +
geom_text(aes(x=90,y=y+.2,label=sprintf("N=%d",N)),
data=df_filt_medians) +
geom_segment(aes(x=price1k_median,y=y-.1,xend=price1k_median,yend=y+.1),
data=df_filt_medians,color="blue",size=2) +
geom_text(aes(x=price1k_median,y=y+.2,label=sprintf("med=%.1f",price1k_median)),
data=df_filt_medians,color="blue") +
geom_segment(aes(x=price1k_mean,y=y-.1,xend=price1k_mean,yend=y+.1),
data=df_filt_medians,color="red",size=2) +
geom_text(aes(x=price1k_mean,y=y-.2,label=sprintf("avg=%.1f",price1k_mean)),
data=df_filt_medians,color="red") +
theme_ridges()

Plota preço vs km
df_filt %>%
mutate(suspicious=negativity_count>4) %>%
ggplot(aes(km1k,price1k,color=brand,group=brand)) +
geom_point(aes(shape=suspicious,size=suspicious)) +
geom_smooth() +
theme_bw() +
ggtitle("Civic, Corolla, Sentra, RJ, particular",
subtitle="km < 80k, price < R$80k, year >= 2016")

attempt: ggvis (tooltips seems buggy, cannot yet add title)
df_filt %>%
ggvis(~km1k, ~price1k, fill=~brand) %>%
group_by(brand) %>%
layer_points() %>%
# hangs
# add_tooltip(function(df) "hello",on="click") %>%
layer_smooths(stroke=~brand)
Using plotly
df_filt %>%
group_by(brand) %>%
mutate(fit = fitted(loess(price1k ~ km1k))) %>%
plot_ly(x = ~km1k, text=~tooltip) %>%
#plot_ly(x = ~km1k, y = ~price1k, group=~brand,
# Hover text:
#text = ~paste(title), color = ~brand, marker=list(size=10)) %>%
add_markers(y = ~price1k, color = ~brand, marker=list(size=10)) %>%
#add_lines(name = "spline", line = list(shape = "spline")) %>%
add_lines(y = ~fit, color = ~brand, showlegend=F) %>%
layout(xaxis = list(showline=T))#%>%
# add_ribbons(data=broom::augment(m),
# ymin = ~.fitted - 1.96 * .se.fit,
# ymax = ~.fitted + 1.96 * .se.fit,
# line = list(color = ~brand),
# fillcolor = ~brand)
LS0tDQp0aXRsZTogIlIgTm90ZWJvb2siDQpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sNCi0tLQ0KDQojIExvYWQgbGlicmFyaWVzDQoNCmBgYHtyIHNldHVwfQ0KbGlicmFyeShtYWdyaXR0cikNCmxpYnJhcnkodGlkeXZlcnNlKQ0KbGlicmFyeShydmVzdCkNCmxpYnJhcnkocGxvdGx5KQ0KbGlicmFyeShnZ3ZpcykNCmxpYnJhcnkoZ2x1ZSkNCmxpYnJhcnkoZ2dyaWRnZXMpDQpgYGANCg0KIyBTY3JhcGluZw0KDQpJbnNwaXJlZCBieTogaHR0cHM6Ly93d3cuYW5hbHl0aWNzdmlkaHlhLmNvbS9ibG9nLzIwMTcvMDMvYmVnaW5uZXJzLWd1aWRlLW9uLXdlYi1zY3JhcGluZy1pbi1yLXVzaW5nLXJ2ZXN0LXdpdGgtaGFuZHMtb24ta25vd2xlZGdlLw0KDQpTZWxlY3RvciBHYWRnZXQgKENocm9tZSBleHRlbnNpb24pOiBodHRwOi8vc2VsZWN0b3JnYWRnZXQuY29tLw0KDQpgYGB7cn0NCmdldF9odG1sX3BhZ2VzIDwtIGZ1bmN0aW9uKHVybHMpIG1hcCh1cmxzLHJlYWRfaHRtbCkNCmBgYA0KDQpDaXZpYywgQ29yb2xsYSwgU2VudHJhLCBSSiwga20gPCA4MDAwMCwgcHJpY2UgPCBSJDgwMDAwLCB5ZWFyID49IDIwMTYsIHBhcnRpY3VsYXINCg0KYGBge3J9DQpvbHhVUkwgPC0gZnVuY3Rpb24oYnJhbmQsbW9kZWwscGFnZSxtYXhQcmljZT04MDAwMCxtYXhLbT04MDAwMCkgZ2x1ZSgiaHR0cDovL3JqLm9seC5jb20uYnIvdmVpY3Vsb3MtZS1wZWNhcy9jYXJyb3Mve2JyYW5kfS97bW9kZWx9P2Y9cCZtZT17bWF4UHJpY2V9Jm89e3BhZ2V9JnBlPXttYXhLbX0mcnM9MzQiKQ0KYGBgDQoNCkNvbnN0cnVjdCBVUkxzDQoNCmBgYHtyfQ0KdXJscyA8LSBtYXBfY2hyKGxpc3QoDQogIGxpc3QoImhvbmRhIiwiY2l2aWMiLDEpLA0KICBsaXN0KCJob25kYSIsImNpdmljIiwyKSwNCiAgbGlzdCgidG95b3RhIiwiY29yb2xsYSIsMSksDQogIGxpc3QoInRveW90YSIsImNvcm9sbGEiLDIpLA0KICBsaXN0KCJ0b3lvdGEiLCJjb3JvbGxhIiwzKSwNCiAgbGlzdCgibmlzc2FuIiwic2VudHJhIiwxKSwNCiAgbGlzdCgibmlzc2FuIiwic2VudHJhIiwyKSksDQogIH5kby5jYWxsKG9seFVSTCwueCkpDQoNCiMgdXJscyA8LSBjKCJodHRwOi8vcmoub2x4LmNvbS5ici92ZWljdWxvcy1lLXBlY2FzL2NhcnJvcy9ob25kYS9jaXZpYz9mPXAmbWU9ODAwMDAmcGU9ODAwMDAmcnM9MzQiDQojICAgICAgICAgICAsImh0dHA6Ly9yai5vbHguY29tLmJyL3ZlaWN1bG9zLWUtcGVjYXMvY2Fycm9zL3RveW90YS9jb3JvbGxhP2Y9cCZtZT04MDAwMCZwZT04MDAwMCZycz0zNCINCiMgICAgICAgICAgICwiaHR0cDovL3JqLm9seC5jb20uYnIvdmVpY3Vsb3MtZS1wZWNhcy9jYXJyb3Mvbmlzc2FuL3NlbnRyYT9mPXAmbWU9ODAwMDAmcGU9ODAwMDAmcnM9MzQiDQojICAgICAgICAgICAjIDJuZCBwYWdlDQojICAgICAgICAgICAsImh0dHA6Ly9yai5vbHguY29tLmJyL3ZlaWN1bG9zLWUtcGVjYXMvY2Fycm9zL2hvbmRhL2NpdmljP2Y9cCZtZT04MDAwMCZvPTImcGU9ODAwMDAmcnM9MzQiDQojICAgICAgICAgICAsImh0dHA6Ly9yai5vbHguY29tLmJyL3ZlaWN1bG9zLWUtcGVjYXMvY2Fycm9zL3RveW90YS9jb3JvbGxhP2Y9cCZtZT04MDAwMCZvPTImcGU9ODAwMDAmcnM9MzQiDQojICAgICAgICAgICAsImh0dHA6Ly9yai5vbHguY29tLmJyL3ZlaWN1bG9zLWUtcGVjYXMvY2Fycm9zL25pc3Nhbi9zZW50cmE/Zj1wJm1lPTgwMDAwJm89MiZwZT04MDAwMCZycz0zNCINCiMgICAgICAgICAgICMgM2QgcGFnZQ0KIyAgICAgICAgICAgLCJodHRwOi8vcmoub2x4LmNvbS5ici92ZWljdWxvcy1lLXBlY2FzL2NhcnJvcy90b3lvdGEvY29yb2xsYT9mPXAmbWU9ODAwMDAmbz0zJnBlPTgwMDAwJnJzPTM0IikNCmBgYA0KDQpHZXQgcGFnZXMNCg0KYGBge3J9DQpwYWdlcyA8LSBnZXRfaHRtbF9wYWdlcyh1cmxzKQ0KYGBgDQoNCkF1eGlsaWFyeSBmdW5jdGlvbnMNCg0KYGBge3J9DQpnZXRfaHRtbF9ocmVmIDwtIGZ1bmN0aW9uKHBhZ2VzLGNzcykgdW5saXN0KG1hcChwYWdlcyx+KGh0bWxfbm9kZXMoLngsY3NzKSAlPiUgaHRtbF9hdHRyKCJocmVmIikpKSkNCg0KZ2V0X2h0bWxfdGV4dCA8LSBmdW5jdGlvbihwYWdlcyxjc3MpIHsNCiAgdGV4dCA8LSB1bmxpc3QobWFwKHBhZ2VzLH5odG1sX3RleHQoaHRtbF9ub2RlcygueCxjc3MpKSkpDQogIHRleHQgJT4lIHN0cl90cmltKCJib3RoIikNCn0NCmBgYA0KDQojIyMgTGlua3MNCg0KYGBge3J9DQp0ZXh0X2xpbmtzIDwtIGdldF9odG1sX2hyZWYocGFnZXMsIi5PTFhhZC1saXN0LWxpbmsiKQ0KdGV4dF9saW5rcyAlPiUgaGVhZA0KYGBgDQoNCiMjIyBDYXIgZGVzY3JpcHRpb24NCg0KYGBge3J9DQp0ZXh0X3RpdGxlcyA8LSBnZXRfaHRtbF90ZXh0KHBhZ2VzLCIuT0xYYWQtbGlzdC10aXRsZSIpDQp0ZXh0X3RpdGxlcyAlPiUgaGVhZA0KYGBgDQoNCiMjIFByZcOnb3MNCg0KYGBge3J9DQpjb252ZXJ0UmVhaXMgPC0gZnVuY3Rpb24ocykgcyAlPiUNCiAgc3RyX3JlcGxhY2VfYWxsKCJbXlxcZF0rIiwiIikgJT4lDQogIGFzLmludGVnZXIoKQ0KYGBgDQoNCmBgYHtyfQ0KdGV4dF9wcmljZXMgPC0gZ2V0X2h0bWxfdGV4dChwYWdlcywiLk9MWGFkLWxpc3QtcHJpY2UiKSAlPiUNCiAgY29udmVydFJlYWlzKC4pDQp0ZXh0X3ByaWNlcyAlPiUgaGVhZA0KYGBgDQoNCiMjIyBLbSwgY8OibWJpbywgbW90b3I6IHZpbmRvIGFzc2ltOiAiMzMuMDAwIGttXG5cdFx0IHwgQ8OibWJpbzogYXV0b23DoXRpY29cblx0XHQgfCBGbGV4Ig0KDQpgYGB7cn0NCmNsZWFyTWtDYW1iaW9Nb3RvciA8LSBmdW5jdGlvbihzKSANCiAgYyhzWzFdICU+JSBzdHJfcmVwbGFjZSgiXFwuIiwiIikgJT4lIHN0cl9yZXBsYWNlKCIgW2tLXW0iLCIiKSwNCiAgICBzWzJdICU+JSBzdHJfcmVwbGFjZSgiQ8OibWJpbzogIiwiIiksDQogICAgc1szXSkNCg0KZGVjb2RlS21DYW1iaW9Nb3RvciA8LSBmdW5jdGlvbihzKSBzICU+JQ0KICBzdHJfcmVwbGFjZV9hbGwoIltcXG5cXHRdIiwiIikgJT4lDQogIHN0cl9zcGxpdCgiXFx8IikgJT4lDQogIG1hcCh+c3RyX3RyaW0oLngsImJvdGgiKSkgJT4lDQogIG1hcCh+Y2xlYXJNa0NhbWJpb01vdG9yKC54KSkNCmBgYA0KDQpgYGB7cn0NCnRleHRfa20gPC0gZ2V0X2h0bWxfdGV4dChwYWdlcywiLmRldGFpbC1zcGVjaWZpYyIpICU+JQ0KICBkZWNvZGVLbUNhbWJpb01vdG9yKC4pDQp0ZXh0X2ttICU+JSBoZWFkDQpgYGANCg0KIyMjIFJlZ2lvbg0KDQpgYGB7cn0NCnRleHRfcmVnaW9uIDwtIGdldF9odG1sX3RleHQocGFnZXMsIi5kZXRhaWwtcmVnaW9uIikgJT4lDQogIHN0cl9yZXBsYWNlX2FsbCgiW1xcc1xcblxcdF0rIiwiICIpDQp0ZXh0X3JlZ2lvbiAlPiUgaGVhZA0KYGBgDQoNCg0KDQojIE1vbnRhZ2VtIGRvIERGDQoNCmBgYHtyfQ0KZGV0ZWN0QnJhbmQgPC0gZnVuY3Rpb24ocykgY2FzZV93aGVuKHN0cl9kZXRlY3QocywiY29yb2xsYSIpfiJjb3JvbGxhIiwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdHJfZGV0ZWN0KHMsInNlbnRyYSIpfiJzZW50cmEiLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0cl9kZXRlY3QocywiY2l2aWMiKX4iY2l2aWMiLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFR+TkFfY2hhcmFjdGVyXykNCmBgYA0KDQoNCmBgYHtyfQ0KZGYgPC0gZGF0YV9mcmFtZSh0aXRsZT1zdHJfc3ViKHRleHRfdGl0bGVzLGVuZD0tNyksDQogICAgICAgICAgICAgICAgIGJyYW5kPWFzLmZhY3RvcihkZXRlY3RCcmFuZChzdHJfdG9fbG93ZXIodGl0bGUpKSksDQogICAgICAgICAgICAgICAgIHJlZ2lvbj1zdHJfcmVwbGFjZSh0ZXh0X3JlZ2lvbiwiIC0gREREIFxcZFxcZCIsIiIpLA0KICAgICAgICAgICAgICAgICBkZGQ9c3RyX3N1Yih0ZXh0X3JlZ2lvbixzdGFydD0tMiksDQogICAgICAgICAgICAgICAgIHllYXI9dGV4dF90aXRsZXMgJT4lIHN0cl9zdWIoc3RhcnQ9LTQpICU+JSBhcy5pbnRlZ2VyKCkgJT4lIGFzLmZhY3RvcigpLA0KICAgICAgICAgICAgICAgICBwcmljZTFrPXJvdW5kKHRleHRfcHJpY2VzLzEwMDAsMSksDQogICAgICAgICAgICAgICAgIGttMWs9cm91bmQoYXMuaW50ZWdlcihtYXBfY2hyKHRleHRfa20sMSkpLzEwMDAsMSksDQogICAgICAgICAgICAgICAgIGNhbWJpbz1hcy5mYWN0b3IobWFwX2Nocih0ZXh0X2ttLDIpKSwNCiAgICAgICAgICAgICAgICAgbW90b3I9YXMuZmFjdG9yKG1hcF9jaHIodGV4dF9rbSwzKSksDQogICAgICAgICAgICAgICAgIGxpbms9dGV4dF9saW5rcykgJVQ+JQ0KICBwcmludA0KYGBgDQoNCiMgU2VudGltZW50IEFuYWx5c2lzDQoNClJlY3Vyc2l2ZWx5IGdldCBjYXIgZGVzY3JpcHRpb24gZnJvbSBsaW5rZWQgcGFnZQ0KDQpgYGB7cn0NCiMgdGhpcyB3aWxsIGhpdCB0aGUgc2VydmVyIG1hbnkgdGltZXMsIHVzZSBzcGFyaW5nbHkNCnBhZ2VzX2Rlc2NyIDwtIGdldF9odG1sX3BhZ2VzKGRmJGxpbmspDQpgYGANCg0KDQpodHRwczovL2dpdGh1Yi5jb20vaGFkbGV5L3J2ZXN0L2lzc3Vlcy8xNzUNCg0KYGBge3J9DQojZnVuY3Rpb24gZGVmaW5pdGlvbg0KaHRtbF90ZXh0X2NvbGxhcHNlIDwtIGZ1bmN0aW9uKHgsIHRyaW0gPSBGQUxTRSwgY29sbGFwc2UgPSAiXG4iKXsNCiAgVXNlTWV0aG9kKCJodG1sX3RleHRfY29sbGFwc2UiKQ0KfQ0KDQpodG1sX3RleHRfY29sbGFwc2UueG1sX25vZGVzZXQgPC0gZnVuY3Rpb24oeCwgdHJpbSA9IEZBTFNFLCBjb2xsYXBzZSA9ICJcbiIpew0KICB2YXBwbHkoeCwgaHRtbF90ZXh0X2NvbGxhcHNlLnhtbF9ub2RlLCBjaGFyYWN0ZXIoMSksIHRyaW0gPSB0cmltLCBjb2xsYXBzZSA9IGNvbGxhcHNlKQ0KfQ0KDQpodG1sX3RleHRfY29sbGFwc2UueG1sX25vZGUgPC0gZnVuY3Rpb24oeCwgdHJpbSA9IEZBTFNFLCBjb2xsYXBzZSA9ICJcbiIpew0KICBwYXN0ZSh4bWwyOjp4bWxfZmluZF9hbGwoeCwgIi4vL3RleHQoKSIpLCBjb2xsYXBzZSA9IGNvbGxhcHNlKQ0KfQ0KYGBgDQoNClVzYW5kbyB3b3JrYXJvdW5kIGRvICJodG1sX3RleHRfY29sbGFwc2UiIHBvaXMgaHRtbF90ZXh0IGVzdGF2YSBjb21lbmRvIG9zIDxicj4gdGFnczoNCg0KYGBge3J9DQpodG1sX25vZGVzKHBhZ2VzX2Rlc2NyW1syNDRdXSwiLk9MWGFkLWRlc2NyaXB0aW9uIikgJT4lIGh0bWxfdGV4dF9jb2xsYXBzZSAlPiUgc3RyX3N1YihlbmQ9NDAwKQ0KYGBgDQoNClZlcnPDo28gZG8gZ2V0X2h0bWxfdGV4dCBjb20gbyB3b3JrYXJvdW5kIHBhcmEgaW5zZXJpciAiXG4iIG5vcyA8YnI+DQoNCmBgYHtyfQ0KZ2V0X2h0bWxfdGV4dF9jb2xsYXBzZSA8LSBmdW5jdGlvbihwYWdlcyxjc3MpIHsNCiAgdGV4dCA8LSB1bmxpc3QobWFwKHBhZ2VzLH5odG1sX3RleHRfY29sbGFwc2UoaHRtbF9ub2RlcygueCxjc3MpKSkpDQogIHRleHQgJT4lIHN0cl90cmltKCJib3RoIikNCn0NCmBgYA0KDQpNb250YW5kbyBub3ZvIERGIGNvbSBkZXNjcmNpw6fDtWVzIChzZWd1aW5kbyBsaW5rIGRvIGFudW5jaW8pLCBsaW1wZXphDQoNCmBgYHtyfQ0KZGZfZGVzY3IgPC0gZGYgJT4lIG11dGF0ZShkZXNjcj11bmxpc3QoZ2V0X2h0bWxfdGV4dF9jb2xsYXBzZShwYWdlc19kZXNjciwiLk9MWGFkLWRlc2NyaXB0aW9uIikpLA0KICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbD11bmxpc3QoZ2V0X2h0bWxfdGV4dChwYWdlc19kZXNjciwiLm1vZGVsIikpKQ0KZGZfZGVzY3JfY2xlYW4gPC0gZGZfZGVzY3IgJT4lDQogIG11dGF0ZShrZXk9cm93X251bWJlcigpLA0KICAgICAgICAgZGVzY3I9ZGVzY3IgJT4lIHN0cl90b19sb3dlcigpICU+JSBzdHJfcmVwbGFjZV9hbGwoIlteWzphbG51bTpdL10iLCIgIikgJT4lDQogICAgICAgICAgIHN0cl9yZXBsYWNlX2FsbCgiXFxzezIsfSIsIiAiKSwNCiAgICAgICAgIG1vZGVsPXN0cl9yZXBsYWNlKG1vZGVsLCJNb2RlbG86W1xcblxcdF0rIiwiIikpDQoNCmBgYA0KDQpSZXBvcnRhIHByZcOnbyBtZWRpYW5vIGUgTiBwb3IgbW9kZWxvLiBQYXJlY2UgcSBtb2RlbG9zIHPDo28gcGFkcm9uaXphZG9zIHRpcG8gRklQRQ0KDQpgYGB7cn0NCmRmX2Rlc2NyX2NsZWFuICU+JQ0KICBncm91cF9ieShtb2RlbCkgJT4lDQogIHN1bW1hcml6ZShwcmljZTFrX21lZGlhbj1tZWRpYW4ocHJpY2UxayksTj1uKCkpICU+JQ0KICBhcnJhbmdlKGRlc2MocHJpY2Uxa19tZWRpYW4pKQ0KYGBgDQoNClBhcmEgY2FkYSBjYXJybyAoaWRlbnRpZmljYWRvIHBvciAia2V5IiksIGxpc3RlIGFzIHBhbGF2cmFzIHVzYWRhcyBuYSBkZXNjcmnDp8Ojby4NCg0KYGBge3J9DQpkZl9kZXNjcl93b3JkcyA8LSBkZl9kZXNjcl9jbGVhbiAlPiUNCiAgc2VsZWN0KGtleSxkZXNjcikgJT4lDQogIG11dGF0ZSh3b3JkPXN0cl9zcGxpdChkZXNjciwiXFxzKyIpKSAlPiUNCiAgc2VsZWN0KC1kZXNjcikgJT4lDQogIHVubmVzdCgpICU+JQ0KICBmaWx0ZXIoc3RyX2xlbmd0aCh3b3JkKT4yLA0KICAgICAgICAgIXN0cl9kZXRlY3Qod29yZCwiXFxkIikpDQpgYGANCg0KRXN0dWRvIGZyZXF1ZW5jaWFsIGRhcyBwYWxhdnJhcw0KDQpgYGB7cn0NCmRmX2Rlc2NyX3dvcmRzX2NvdW50ZWQgPC0gZGZfZGVzY3Jfd29yZHMgJT4lDQogIGNvdW50KHdvcmQsc29ydD1UKSAlVD4lIHByaW50DQpgYGANCg0KU2FsdmFtb3MgZW0gZXhjZWwgcGFyYSBtYW51YWxtZW50ZSBtYXJjYXJtb3MgcGFsYXZyYXMgY29tIHNlbnRpbWVudG8gbmVnYXRpdm8NCg0KYGBge3J9DQpkZl9kZXNjcl93b3Jkc19jb3VudGVkICU+JQ0KICBhcy5kYXRhLmZyYW1lKCkgJT4lDQogIHhsc3g6OndyaXRlLnhsc3goImRmX2Rlc2NyX3dvcmRzX2NvdW50ZWQueGxzIiwgcm93Lm5hbWVzPUYpDQpgYGANCg0KTGUgYXJxdWl2byBjb20gcGFsYXZyYXMgY29tIHNlbnRpbWVudG8gbmVnYXRpdm8NCg0KYGBge3J9DQpkZl90cmV0YXMgPC0gcmVhZF9jc3YoInN1c3BpY2lvdXMgd29yZHMuY3N2IikgJVQ+JSBwcmludA0KYGBgDQoNCkFjaGEgYW51bmNpb3MgcSBjb250ZW0gcGFsYXZyYSBuZWdhdGl2YQ0KDQpgYGB7cn0NCmRmX2Rlc2NyX3dvcmRzX25lZ2F0aXZlIDwtIGRmX2Rlc2NyX3dvcmRzICU+JQ0KICBzZW1pX2pvaW4oZGZfdHJldGFzLGJ5PSJ3b3JkIikgJT4lDQogIGdyb3VwX2J5KGtleSkgJT4lDQogIHN1bW1hcml6ZShuZWdhdGl2aXR5X2NvdW50PW4oKSwNCiAgICAgICAgICAgIGJhZF93b3Jkcz1wYXN0ZTAod29yZCxjb2xsYXBzZT0iLCIpKSAlVD4lIA0KICBwcmludA0KYGBgDQoNCkV4cGFuZGUgZGYgZGUgYW7Dum5jaW9zIGNvbSBhbsOhbGlzZSBkZSBzZW50aW1lbnRvDQoNCmBgYHtyfQ0KZGZfZGVzY3JfY2xlYW5fc2VudGltZW50IDwtIGRmX2Rlc2NyX2NsZWFuICU+JQ0KICBsZWZ0X2pvaW4oZGZfZGVzY3Jfd29yZHNfbmVnYXRpdmUsYnk9ImtleSIpICU+JQ0KICBtdXRhdGUobmVnYXRpdml0eV9jb3VudD1pZl9lbHNlKGlzLm5hKG5lZ2F0aXZpdHlfY291bnQpLDBMLG5lZ2F0aXZpdHlfY291bnQpKSAlPiUNCiAgYXJyYW5nZShkZXNjKG5lZ2F0aXZpdHlfY291bnQpKSAlVD4lDQogIHByaW50DQpgYGANCg0KDQoNCg0KIyBQbG90cw0KDQoNCmBgYHtyfQ0KZGZfZmlsdCA8LSBkZl9kZXNjcl9jbGVhbl9zZW50aW1lbnQgJT4lDQogIGZpbHRlcihwcmljZTFrPjQwLCFpcy5uYShicmFuZCkseWVhcj09MjAxNikgJT4lDQogIG11dGF0ZSh0b29sdGlwPWdsdWUoIjxhIGhyZWY9e2xpbmt9Pnt0aXRsZX1cbntyZWdpb259XG48L2E+IikpDQpgYGANCg0KYGBge3J9DQpkZl9maWx0ICU+JQ0KICBnZ3Bsb3QoYWVzKHg9bmVnYXRpdml0eV9jb3VudCkpICsNCiAgZ2VvbV9iYXIoc3RhdD0iY291bnQiKQ0KYGBgDQoNCg0KDQpgYGB7cn0NCmZ1bl9sZW5ndGhfeSA8LSBmdW5jdGlvbih4KSBkYXRhLmZyYW1lKHk9bWVkaWFuKHgpLGxhYmVsPSBwYXN0ZTAoIk49IiwgbGVuZ3RoKHgpKSkNCg0KZGZfZmlsdCAlPiUNCiAgbXV0YXRlKGJyYW5kPWZjdF9yZW9yZGVyKGJyYW5kLHByaWNlMWssLmRlc2M9VCksDQogICAgICAgICBzdXNwaWNpb3VzPW5lZ2F0aXZpdHlfY291bnQ+MSkgJT4lDQogIGFycmFuZ2Uoc3VzcGljaW91cykgJT4lICMgc28gdHJ1ZSBpcyBkcmF3biBsYXN0DQogICNtdXRhdGUoc3VzcGljaW91cz1hcy5mYWN0b3Ioc3VzcGljaW91cykgJT4lIGZjdF9pbm9yZGVyKCkgJT4lIGZjdF9yZXYoKSkgJT4lDQogIGdncGxvdChhZXMoeD1icmFuZCx5PXByaWNlMWssZmlsbD1icmFuZCkpICsNCiAgI2dlb21fdmlvbGluKGFscGhhPS4yNSkgKw0KICBnZW9tX2JveHBsb3Qobm90Y2ggPSBULCBzaG93LmxlZ2VuZCA9IEYpICsNCiAgc2NhbGVfZmlsbF9kaXNjcmV0ZShndWlkZSA9ICJub25lIikgKw0KICBzdGF0X3N1bW1hcnkoZnVuLmRhdGEgPSBmdW5fbGVuZ3RoX3ksDQogICAgICAgICAgICAgICBnZW9tID0gInRleHQiLCB2anVzdCA9IC41LGhqdXN0PTIuNSwgc2l6ZSA9IDQpICsNCiAgZ2VvbV9qaXR0ZXIoYWVzKGNvbG9yPXN1c3BpY2lvdXMsc2hhcGU9c3VzcGljaW91cyksYWxwaGE9LjUsd2lkdGg9LjEsc2l6ZT0zKSArDQogIHNjYWxlX3NoYXBlX21hbnVhbCh2YWx1ZXM9YygxNiwxNykpICsNCiAgc2NhbGVfY29sb3VyX21hbnVhbCh2YWx1ZXMgPSBjKCJncmF5IiwicmVkIikpICsNCiAgdGhlbWVfYncoKSArDQogIGdndGl0bGUoIkNpdmljLCBDb3JvbGxhLCBTZW50cmEsIFJKLCBwYXJ0aWN1bGFyIiwNCiAgICAgICAgICBzdWJ0aXRsZT0ia20gPCA4MGssIHByaWNlIDwgUiQ4MGssIHllYXIgPj0gMjAxNiIpICMgKw0KICAjdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gIm5vbmUiKQ0KYGBgDQoNCmBgYHtyfQ0KZGZfZmlsdF9tZWRpYW5zIDwtIGRmX2ZpbHQgJT4lDQogIGdyb3VwX2J5KGJyYW5kKSAlPiUNCiAgc3VtbWFyaXplKE49bigpLA0KICAgICAgICAgICAgcHJpY2Uxa19tZWRpYW49bWVkaWFuKHByaWNlMWspLA0KICAgICAgICAgICAgcHJpY2Uxa19tZWFuPW1lYW4ocHJpY2UxaykpICU+JQ0KICBhcnJhbmdlKGRlc2MocHJpY2Uxa19tZWRpYW4pKSAlPiUNCiAgbXV0YXRlKHk9cm93X251bWJlcigpKQ0KICANCg0KZGZfZmlsdCAlPiUNCiAgbXV0YXRlKGJyYW5kPWZjdF9yZW9yZGVyKGJyYW5kLHByaWNlMWssLmRlc2M9VCkpICU+JQ0KICBnZ3Bsb3QoYWVzKHByaWNlMWssIGJyYW5kLGZpbGw9YnJhbmQpKSArDQogIGdlb21fZGVuc2l0eV9yaWRnZXMoKSArDQogIGdlb21fdGV4dChhZXMoeD05MCx5PXkrLjIsbGFiZWw9c3ByaW50ZigiTj0lZCIsTikpLA0KICAgICAgICAgICAgZGF0YT1kZl9maWx0X21lZGlhbnMpICsNCiAgZ2VvbV9zZWdtZW50KGFlcyh4PXByaWNlMWtfbWVkaWFuLHk9eS0uMSx4ZW5kPXByaWNlMWtfbWVkaWFuLHllbmQ9eSsuMSksDQogICAgICAgICAgICBkYXRhPWRmX2ZpbHRfbWVkaWFucyxjb2xvcj0iYmx1ZSIsc2l6ZT0yKSArDQogIGdlb21fdGV4dChhZXMoeD1wcmljZTFrX21lZGlhbix5PXkrLjIsbGFiZWw9c3ByaW50ZigibWVkPSUuMWYiLHByaWNlMWtfbWVkaWFuKSksDQogICAgICAgICAgICBkYXRhPWRmX2ZpbHRfbWVkaWFucyxjb2xvcj0iYmx1ZSIpICsNCiAgZ2VvbV9zZWdtZW50KGFlcyh4PXByaWNlMWtfbWVhbix5PXktLjEseGVuZD1wcmljZTFrX21lYW4seWVuZD15Ky4xKSwNCiAgICAgICAgICAgICAgIGRhdGE9ZGZfZmlsdF9tZWRpYW5zLGNvbG9yPSJyZWQiLHNpemU9MikgKw0KICBnZW9tX3RleHQoYWVzKHg9cHJpY2Uxa19tZWFuLHk9eS0uMixsYWJlbD1zcHJpbnRmKCJhdmc9JS4xZiIscHJpY2Uxa19tZWFuKSksDQogICAgICAgICAgICBkYXRhPWRmX2ZpbHRfbWVkaWFucyxjb2xvcj0icmVkIikgKw0KICB0aGVtZV9yaWRnZXMoKQ0KYGBgDQoNCg0KUGxvdGEgcHJlw6dvIHZzIGttDQoNCmBgYHtyfQ0KZGZfZmlsdCAlPiUNCiAgbXV0YXRlKHN1c3BpY2lvdXM9bmVnYXRpdml0eV9jb3VudD40KSAlPiUNCiAgZ2dwbG90KGFlcyhrbTFrLHByaWNlMWssY29sb3I9YnJhbmQsZ3JvdXA9YnJhbmQpKSArDQogIGdlb21fcG9pbnQoYWVzKHNoYXBlPXN1c3BpY2lvdXMsc2l6ZT1zdXNwaWNpb3VzKSkgKw0KICBnZW9tX3Ntb290aCgpICsNCiAgdGhlbWVfYncoKSArDQogIGdndGl0bGUoIkNpdmljLCBDb3JvbGxhLCBTZW50cmEsIFJKLCBwYXJ0aWN1bGFyIiwNCiAgICAgICAgICBzdWJ0aXRsZT0ia20gPCA4MGssIHByaWNlIDwgUiQ4MGssIHllYXIgPj0gMjAxNiIpDQpgYGANCg0KYXR0ZW1wdDogZ2d2aXMgKHRvb2x0aXBzIHNlZW1zIGJ1Z2d5LCBjYW5ub3QgeWV0IGFkZCB0aXRsZSkNCg0KYGBge3J9DQpkZl9maWx0ICU+JQ0KICBnZ3Zpcyh+a20xaywgfnByaWNlMWssIGZpbGw9fmJyYW5kKSAlPiUNCiAgZ3JvdXBfYnkoYnJhbmQpICU+JQ0KICBsYXllcl9wb2ludHMoKSAlPiUNCiAgIyBoYW5ncw0KICAjIGFkZF90b29sdGlwKGZ1bmN0aW9uKGRmKSAiaGVsbG8iLG9uPSJjbGljayIpICU+JQ0KICBsYXllcl9zbW9vdGhzKHN0cm9rZT1+YnJhbmQpIA0KYGBgDQoNClVzaW5nIHBsb3RseQ0KDQpgYGB7cn0NCmRmX2ZpbHQgJT4lDQogIGdyb3VwX2J5KGJyYW5kKSAlPiUNCiAgbXV0YXRlKGZpdCA9IGZpdHRlZChsb2VzcyhwcmljZTFrIH4ga20xaykpKSAlPiUNCiAgcGxvdF9seSh4ID0gfmttMWssIHRleHQ9fnRvb2x0aXApICU+JQ0KICAjcGxvdF9seSh4ID0gfmttMWssIHkgPSB+cHJpY2UxaywgZ3JvdXA9fmJyYW5kLA0KICAjIEhvdmVyIHRleHQ6DQogICN0ZXh0ID0gfnBhc3RlKHRpdGxlKSwgY29sb3IgPSB+YnJhbmQsIG1hcmtlcj1saXN0KHNpemU9MTApKSAlPiUNCiAgYWRkX21hcmtlcnMoeSA9IH5wcmljZTFrLCBjb2xvciA9IH5icmFuZCwgbWFya2VyPWxpc3Qoc2l6ZT0xMCkpICU+JQ0KICAjYWRkX2xpbmVzKG5hbWUgPSAic3BsaW5lIiwgbGluZSA9IGxpc3Qoc2hhcGUgPSAic3BsaW5lIikpICU+JQ0KICBhZGRfbGluZXMoeSA9IH5maXQsIGNvbG9yID0gfmJyYW5kLCBzaG93bGVnZW5kPUYpICU+JQ0KICBsYXlvdXQoeGF4aXMgPSBsaXN0KHNob3dsaW5lPVQpKSMlPiUNCiAgIyBhZGRfcmliYm9ucyhkYXRhPWJyb29tOjphdWdtZW50KG0pLA0KICAjICAgICAgICAgICAgIHltaW4gPSB+LmZpdHRlZCAtIDEuOTYgKiAuc2UuZml0LA0KICAjICAgICAgICAgICAgIHltYXggPSB+LmZpdHRlZCArIDEuOTYgKiAuc2UuZml0LA0KICAjICAgICAgICAgICAgIGxpbmUgPSBsaXN0KGNvbG9yID0gfmJyYW5kKSwNCiAgIyAgICAgICAgICAgICBmaWxsY29sb3IgPSB+YnJhbmQpDQpgYGANCg0KDQoNCg==